{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenizer 基本使用"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "from transformers import AutoTokenizer"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": "sen = \"每天需要去学习,这样才能不被淘汰,年龄大了又怎样?!\"",
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step1 加载与保存"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T13:50:54.221892Z",
     "start_time": "2025-09-05T13:50:53.004287Z"
    }
   },
   "source": [
    "# 从HuggingFace加载，输入模型名称，即可加载对应的的分词器\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"uer/roberta-base-finetuned-dianping-chinese\")\n",
    "tokenizer"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertTokenizerFast(name_or_path='uer/roberta-base-finetuned-dianping-chinese', vocab_size=21128, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=True),  added_tokens_decoder={\n",
       "\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t100: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t101: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t102: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t103: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "}"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 123
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-05T13:50:56.515737Z",
     "start_time": "2025-09-05T13:50:56.501299Z"
    }
   },
   "source": [
    "# tokenizer 保存到本地\n",
    "tokenizer.save_pretrained(\"./roberta_tokenizer\")"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./roberta_tokenizer\\\\tokenizer_config.json',\n",
       " './roberta_tokenizer\\\\special_tokens_map.json',\n",
       " './roberta_tokenizer\\\\vocab.txt',\n",
       " './roberta_tokenizer\\\\added_tokens.json',\n",
       " './roberta_tokenizer\\\\tokenizer.json')"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 124
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# 从本地加载tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"./roberta_tokenizer/\")\n",
    "tokenizer"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step2 句子分词"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "tokens = tokenizer.tokenize(sen)\n",
    "tokens"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step3 查看词典"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "tokenizer.vocab"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "tokenizer.vocab_size"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step4 索引转换"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# 将词序列转换为id序列\n",
    "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "ids"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# 将id序列转换为token序列\n",
    "tokens = tokenizer.convert_ids_to_tokens(ids)\n",
    "tokens"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# 将token序列转换为string\n",
    "str_sen = tokenizer.convert_tokens_to_string(tokens)\n",
    "str_sen"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  更便捷的实现方式"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# 将字符串转换为id序列，又称之为编码\n",
    "ids = tokenizer.encode(sen, add_special_tokens = True)\n",
    "ids"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# 将id序列转换为字符串，又称之为解码\n",
    "str_sen = tokenizer.decode(ids, skip_special_tokens = False)\n",
    "str_sen"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step5 填充与截断"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# 填充\n",
    "ids = tokenizer.encode(sen, padding = \"max_length\", max_length = 15)\n",
    "ids"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# 截断\n",
    "ids = tokenizer.encode(sen, max_length = 5, truncation = True)\n",
    "ids"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step6 其他输入部分"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "ids = tokenizer.encode(sen, padding = \"max_length\", max_length = 15)\n",
    "ids"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "attention_mask = [1 if idx != 0 else 0 for idx in ids]\n",
    "token_type_ids = [0] * len(ids)\n",
    "ids, attention_mask, token_type_ids"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step7 快速调用方式"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "inputs = tokenizer.encode_plus(sen, padding = \"max_length\", max_length = 15)\n",
    "inputs"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "inputs = tokenizer(sen, padding = \"max_length\", max_length = 15)\n",
    "inputs"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step8 处理batch数据"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "sens = [\"每天需要去学习\",\n",
    "        \"这样才能不被淘汰\",\n",
    "        \",年龄大了又怎样?!\"]\n",
    "res = tokenizer(sens)\n",
    "res"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "%%time\n",
    "# 单条循环处理\n",
    "for i in range(1000):\n",
    "\ttokenizer(sen)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "%%time\n",
    "# 处理batch数据\n",
    "res = tokenizer([sen] * 1000)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "tokenizer"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fast / Slow Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": "sen = \"哪里的天空好清澈!\"",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "fast_tokenizer = AutoTokenizer.from_pretrained(\"uer/roberta-base-finetuned-dianping-chinese\")\n",
    "fast_tokenizer"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "slow_tokenizer = AutoTokenizer.from_pretrained(\"uer/roberta-base-finetuned-dianping-chinese\", use_fast = False)\n",
    "slow_tokenizer"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "%%time\n",
    "# 单条循环处理\n",
    "for i in range(10000):\n",
    "\tfast_tokenizer(sen)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "%%time\n",
    "# 单条循环处理\n",
    "for i in range(10000):\n",
    "\tslow_tokenizer(sen)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "%%time\n",
    "# 处理batch数据\n",
    "res = fast_tokenizer([sen] * 10000)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "%%time\n",
    "# 处理batch数据\n",
    "res = slow_tokenizer([sen] * 10000)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "inputs = fast_tokenizer(sen, return_offsets_mapping = True)\n",
    "inputs"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "inputs.word_ids()"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": "inputs = slow_tokenizer(sen)",
   "outputs": [],
   "execution_count": null
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特殊Tokenizer的加载"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "from transformers import AutoTokenizer"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "# 新版本的transformers（>4.34），加载 THUDM/chatglm 会报错，因此这里替换为了天宫的模型\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"Skywork/Skywork-o1-Open-PRM-Qwen-2.5-7B\", trust_remote_code = True)\n",
    "tokenizer"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": "tokenizer.save_pretrained(\"skywork_tokenizer\")",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": "tokenizer = AutoTokenizer.from_pretrained(\"skywork_tokenizer\", trust_remote_code = True)",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "tokenizer.decode(tokenizer.encode(sen))"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
